const puppeteer = require('puppeteer');
const MongoClient = require('mongodb').MongoClient;
const ObjectId = require('mongodb').ObjectId;

const mongo_url = "mongodb://egg_cnode:bojun1234@124.223.34.189:27127/egg_cnode?authSource=egg_cnode";


const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

async function main() {
  browser = await puppeteer.launch({
    headless: false,
    ignoreHTTPSErrors: true,
    args: ['--no-sandbox', `--proxy-server=http://u5329.b5.t.16yun.cn:6460`]
  });
  const page = (await browser.pages())[0];
  await page.authenticate({'username': '16GADNHJ', 'password': '936441'});
  await page.setViewport({width: 1500, height: 0});

  const client = await MongoClient.connect(mongo_url, {useNewUrlParser: true, useUnifiedTopology: true});
  const db = client.db('egg_cnode');

  let sub_url = '';
  let tab = 'knowledge';
  let sub_tab = '果壳';
  const fileName = `${tab}_${sub_tab}.json`;
  const source = '果壳' + (new Date(+new Date() + 1000 * 3600 * 8)).toISOString().split('T')[0];
  const real_author_name = '科学人';


  page.on('pageerror', msg => {
    console.log(msg);
  });

  let url = `https://www.guokr.com/beta/proxy/science_api/articles?retrieve_type=by_category&page=2`;
  await page.goto(url, {waitUntil: 'domcontentloaded'});

};


main();


